# -*- coding: utf-8 -*-
import scrapy
import socket
import datetime
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, Join
from ken_properties.items import KenPropertiesItem


class EasySpider(CrawlSpider):
    name = 'easy'
    allowed_domains = ['localhost']
    start_urls = ['http://localhost:9312/properties/index_000000.html']

    # TODO 各种提取器的使用？
    rules = (
        Rule(LinkExtractor(restrict_xpaths='//*[contains(@class,"next")]')),
        Rule(LinkExtractor(
            restrict_xpaths='//*[@itemprop="url"]'), callback='parse_item'),
    )

    def parse_item(self, response):
        """
        docstring here
            :param self: 
            :param response: 

        TODO 不好使？？ scrapy check easy  
        @url http://loaclhost:9312/properties/property_000000.html
        @returns items 1
        @scrapes title price description address image_urls
        @scrapes url project spider server date
        """
        print("==============parse_item======================")
        item_loader = ItemLoader(item=KenPropertiesItem(), response=response)
        item_loader.add_xpath('title', '//*[@itemprop="name"][1]/text()',
                              MapCompose(str.strip, str.title))
        item_loader.add_xpath('price', './/*[@itemprop="price"][1]/text()',
                              MapCompose(lambda i: i.replace(',', ''), float),
                              re='[,.0-9]+')
        item_loader.add_xpath('description', '//*[@itemprop="description"][1]/text()',
                              MapCompose(str.strip), Join())
        item_loader.add_xpath('address',
                              '//*[@itemtype="http://schema.org/Place"][1]/text()',
                              MapCompose(str.strip))
        item_loader.add_xpath('image_urls', '//*[@itemprop="image"][1]/@src',
                              MapCompose(lambda i: response.urljoin(i)))

        # Housekeeping fields
        item_loader.add_value('url', response.url)
        item_loader.add_value('project', self.settings.get('BOT_NAME'))
        item_loader.add_value('spider', self.name)
        item_loader.add_value('server', socket.gethostname())
        item_loader.add_value('date', datetime.datetime.now())

        print("=================parse_item===================")
        return item_loader.load_item()
